package com.bigdata.dataflow;

import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.math.BigInteger;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Collection;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;

import org.ansj.app.keyword.KeyWordComputer;
import org.ansj.app.keyword.Keyword;
import org.junit.Test;

import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.dtruth.dataflow.bolt.BodyReformatBolt;
import com.dtruth.dataflow.dao.HBaseDAO;
import com.dtruth.dataflow.util.HtmlRegexpUtil;
import com.dtruth.dataflow.util.SimHashUtil;
import com.dtruth.dataflow.util.StringUtil;

public class DataValidate {

	@Test
	public void validDomain() {
		try {
			HBaseDAO hbase = new HBaseDAO();
			String rowKey = StringUtil
					.getMD5("http://news.gz.fang.com/2014-12-30/14515175.htm");
			Map<String, Object> map = hbase.selectRow("crawled", rowKey);
			String body = map.get("body").toString();
			String begin = "<!--wirelessHiddenContentBegin-->";
			String end = "<!--wirelessHiddenContentEnd-->";
			System.out.println(body);
			int b = body.indexOf(begin);
			int e = body.indexOf(end);
			String subStr = body.substring(b, e + end.length());
			System.out.println(body.replaceAll(subStr, ""));

		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
	}

	@Test
	public void testLabels() {
		try {
			HBaseDAO hbase = new HBaseDAO();
			String tablename = "crawled";
			// String rowKey = StringUtil
			// .MD5("http://guba.sina.com.cn/?s=thread&tid=91126454&bid=2");
			// System.out.println(rowKey);
			String rowKey = "d13de68f6e8c563e23d48c28a59765aa";
			Map<String, Object> map = hbase.selectRow(tablename, rowKey);
			KeyWordComputer kwc = new KeyWordComputer(20);
			String title = map.get("title").toString();
			// String tags = map.get("tags") == null ? "" :
			// map.get("tags").toString();
			String content = map.get("article").toString();
			Collection<Keyword> result = kwc.computeArticleTfidf(title + ","
					+ title + "," + title, content);
			System.out.println(result);
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
	}

	@Test
	public void testBodyTagBolt() {
		try {
			HBaseDAO hbase = new HBaseDAO();
			String rowKey = "a5b45727d0a8f2147d0d3fd95bb80fab";
			Map<String, Object> map = hbase.selectRow("crawled", rowKey);
			String body = map.get("body").toString();
			body = HtmlRegexpUtil.getValidContent(body);
			System.out.println(body);
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
	}

	@Test
	public void testRvDuplt() {
		HBaseDAO hbase = new HBaseDAO();
		try {
			// String rowKey1 = StringUtil
			// .getMD5("http://www.nbd.com.cn/articles/2015-01-21/892276.html");
			String rowKey1 = " 6a4f7252c8f25bc84668b7d35f58cc3e";
			Map<String, Object> map1 = hbase.selectRow("crawled", rowKey1);
			String t1 = map1.get("title").toString();
			String a1 = map1.get("body").toString();
			System.out.println(map1.get("url").toString());
			a1 = HtmlRegexpUtil.getValidContent(a1);
			BigInteger tbi1 = SimHashUtil.getSimHash(t1, 64);
			BigInteger abi1 = SimHashUtil.getSimHash(a1, 64);
			// String rowKey2 = StringUtil
			// .getMD5("http://news.pedaily.cn/201501/20150121377347.shtml");
			String rowKey2 = "41bdc079e2581edb640bfe376aaaf979";
			Map<String, Object> map2 = hbase.selectRow("crawled", rowKey2);
			String t2 = map2.get("title").toString();
			String a2 = map2.get("body").toString();
			System.out.println(map2.get("url").toString());
			a2 = HtmlRegexpUtil.getValidContent(a2);
			BigInteger tbi2 = SimHashUtil.getSimHash(t2, 64);
			// BigInteger tbi2 = new BigInteger("00000");
			BigInteger abi2 = SimHashUtil.getSimHash(a2, 64);
			System.out.println(abi2);
			System.out.println("title distance:"
					+ SimHashUtil.hammingDistance(tbi1, tbi2));
			System.out.println("content distance:"
					+ SimHashUtil.hammingDistance(abi1, abi2));
			// System.out.println("distance:" + SimHashUtil.getDistance(bi1,
			// bi2));
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
	}

	@Test
	public void testRvTextDuplt() {
		String txt1 = readFromTxt("F:/test/1.txt");
		BigInteger bi1 = SimHashUtil.getSimHash(txt1, 64);
		System.out.println(bi1);
		String txt2 = readFromTxt("F:/test/2.txt");
		BigInteger bi2 = SimHashUtil.getSimHash(txt2, 64);
		System.out.println(bi2);
		System.out.println("distance:" + SimHashUtil.hammingDistance(bi1, bi2));
		// System.out.println("distance:" + SimHashUtil.getDistance(bi1, bi2));
	}

	@Test
	public void testMD5() {
		try {
			HBaseDAO hbase = new HBaseDAO();
			String rowKey2 = "ae8dc9ec2a4cea4697ac940b648d33f1";
			Map<String, Object> map2 = hbase.selectRow("crawled", rowKey2);
			String t = map2.get("title").toString();
			String a = map2.get("article").toString();
			long start = System.currentTimeMillis();
			String md5 = StringUtil.getMd5(t + "," + a);
			System.out
					.println(md5 + ":" + (System.currentTimeMillis() - start));
			start = System.currentTimeMillis();
			BigInteger simhash = SimHashUtil.getSimHash(t + "," + a);
			System.out.println(simhash + ":"
					+ (System.currentTimeMillis() - start));
			start = System.currentTimeMillis();
			simhash = SimHashUtil.getSimHash(t + "," + a);
			System.out.println(simhash + ":"
					+ (System.currentTimeMillis() - start));
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
	}

	@Test
	public void testBigInteger() {
		BigInteger bi1 = new BigInteger("0");
		BigInteger bi2 = new BigInteger("0");
		long start = System.currentTimeMillis();
		System.out.println(bi1.equals(bi2));
		System.out.println((System.currentTimeMillis() - start) + "ms");
		start = System.currentTimeMillis();
		int dis = SimHashUtil.hammingDistance(bi1, bi2);
		System.out.println(dis == 0);
		System.out.println((System.currentTimeMillis() - start) + "ms");

	}

	@Test
	public void testReformat() {
		try {
			HBaseDAO hbase = new HBaseDAO();
			// String rowKey = StringUtil
			// .getMd5("http://www.cs.com.cn/ssgs/gsxw/201501/t20150106_4609633.html");

			String rowKey = "f8401285a6ae18673d9f67b6809b726b";
			System.out.println(rowKey);
			Map<String, Object> map = hbase.selectRow("crawled", rowKey);
			String htmlStr = map.get("body").toString();
			System.out.println(htmlStr);
			htmlStr = htmlStr.replaceAll("<em>浏览原图</em>", "");
			htmlStr = HtmlRegexpUtil.removeComment(htmlStr);
			htmlStr = HtmlRegexpUtil.removeScript(htmlStr);
			htmlStr = HtmlRegexpUtil.removeStyle(htmlStr);
			Set<String> keepTags = new HashSet<String>();
			keepTags.add("p");
			keepTags.add("img");
			keepTags.add("br");
			keepTags.add("div");
			keepTags.add("ol");
			keepTags.add("ul");
			keepTags.add("li");
			keepTags.add("ul");
			keepTags.add("table");
			keepTags.add("tr");
			keepTags.add("td");
			htmlStr = HtmlRegexpUtil.removePartTags(htmlStr, keepTags);
			htmlStr = HtmlRegexpUtil.removeSpace(htmlStr);
			Set<String> keepTagAttr = new HashSet<String>();
			keepTagAttr.add("img");
			keepTags.removeAll(keepTagAttr);
			for (String keepTag : keepTags) {
				htmlStr = HtmlRegexpUtil.removeTagAllAttrib(htmlStr, keepTag);
			}
			htmlStr = htmlStr.replaceAll("<div\\s*></div>", "");
			htmlStr = htmlStr.replaceAll("<p\\s*>&nbsp;</p>", "");
			htmlStr = htmlStr.replaceAll("<p\\s*></p>", "");
			System.out.println(htmlStr.trim());
		} catch (IOException e) {
			e.printStackTrace();
		}

	}

	@Test
	public void testBodyReformatBolt() {
		try {
			BodyReformatBolt brb = new BodyReformatBolt();
			HBaseDAO hbase = new HBaseDAO();
			String rowKey = "8466de0dcd36d327bd513f2a972c38eb";
			Map<String, Object> map = hbase.selectRow("crawled", rowKey);
			System.out.println(map.get("body"));
			System.out.println(brb.process(map));
		} catch (IOException e) {
			e.printStackTrace();
		}
	}

	@Test
	public void testBodyReformatBoltByTxt() {
		BodyReformatBolt brb = new BodyReformatBolt();
		Map<String, Object> map = new HashMap<String, Object>();
		// String body =
		// "<p>\r\n\t\t\t\t\t\t                    \r\n                        \r\n\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t                        \r\n                        \r\n                        \r\n                        \r\n                        \r\n                    \r\n                        \r\n                        <div  >\r\n                        \t\r\n                            责编：武帅\r\n                        </div>\r\n                        </p>";
		String body = readFromTxt("F:\\test\\body.html");
		map.put("body", JSONObject.parseObject(body).getString("body"));
		map.put("domain", "cebnet.com.cn");
		String result = brb.process(map);
		System.out.println(result);
	}

	public String readFromTxt(String fileName) {
		StringBuffer sb = new StringBuffer();
		try {
			BufferedReader br = new BufferedReader(new InputStreamReader(
					new FileInputStream(fileName), "utf-8"));
			String line = null;
			while ((line = br.readLine()) != null) {
				sb.append(line);
			}
			br.close();
			// fr.close();
		} catch (IOException e) {
			e.printStackTrace();
		}
		return sb.toString();
	}

	@Test
	public void testDateFormat() {
		HBaseDAO hbase = new HBaseDAO();
		String tablename = "crawled";
		String rowKey = StringUtil.getMD5("http://www.iyiou.com/p/15527");
		try {
			Map<String, Object> map = hbase.selectRow(tablename, rowKey);
			String domain = null;
			if (map.get("urllib") != null) {
				int start = map.get("urllib").toString().indexOf(":", 7);
				int end = map.get("urllib").toString().indexOf(":", 15);
				domain = map.get("urllib").toString().substring(start + 1, end);
			}
			String dateformat = "";
			String pubdate = map.get("pubdate").toString();
			SimpleDateFormat sdf = new SimpleDateFormat(dateformat);

			Date d = sdf.parse(pubdate.trim());
			System.out.println(d.getTime());

		} catch (IOException | ParseException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
	}

	@Test
	public void testDateformat() {
		String d = "15年02月04日 09:31";
		SimpleDateFormat sdf = new SimpleDateFormat("yy年MM月dd日 HH:mm");
		SimpleDateFormat sdf_01 = new SimpleDateFormat("yyyy年MM月dd日 HH:mm");
		try {
			System.out.println(sdf_01.format(sdf.parse(d)));
		} catch (ParseException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
	}

	@Test
	public void testPicProcess() {
		try {
			String rowKey = "c58f75a96a2f05e25b71088d49747eb3";
			HBaseDAO hbase = new HBaseDAO();
			Map<String, Object> map = hbase.selectRow("crawled", rowKey);
			Object obj = map.get("thumbnail");
			if (obj != null) {
				System.out.println(obj);
				JSONArray jarry = JSONObject.parseArray(obj.toString());
				JSONArray result = new JSONArray();
				for (int i = 0; i < jarry.size(); i++) {
					String tb = jarry.getString(i);
					if (tb.indexOf("feheadline") == -1)
						result.add(tb);
				}
				map.put("thumbnail", result);
				System.out.println(result);
			}

		} catch (IOException e) {
			e.printStackTrace();
		}
	}
}
